import json
import os.path

from html import unescape
import py3db.mysql
from lxml import etree
from fetch.duplicate_removal import DuplicateRemoval
from config.db import DB_CONFIG
from parse.filter import Filter
from jsonpath import jsonpath
from util.image_util import ImageUtil


class Parse:
    def __init__(
            self,
            item={},
            app_info=None,
            next_url_rule=None,
            next_layout_info=None):

        self.parse_tool = None  #
        self.content = None
        self.item = item  # 要解析的项
        self.field_item = dict()  # 保存解析的项
        self.data_list = []
        self.element_list = []
        self.data = dict()
        # 值为xpath或jsonpath
        self.next_url_rule = next_url_rule
        self.next_layout_info = next_layout_info
        self.app_info = app_info
        self.filter = Filter()
        self.db = py3db.mysql.MySql(DB_CONFIG['host'], DB_CONFIG['user'], DB_CONFIG['password'], app_info[1])
        self.db.operation_database(f"use {self.app_info[1]}")
        self.duplicate = DuplicateRemoval(app_code=app_info[1])
        self.img_util = ImageUtil(app_code=app_info[1], header=app_info[5])

    def parse(self, content, parse_method='xpath', url=None):
        if parse_method == "xpath":
            self.content = etree.HTML(content)
        elif parse_method == "json":
            self.content = json.loads(content)

        for table, value in self.item.items():
            # print("value:", value)
            self.field_item = value
            # print("value:", value)
            if self.field_item.get('list'):
                self.parse_list(table, parse_method, url)
                # print("爬取列表")
            else:
                # print("爬取内容")
                self.data = {}
                self.parse_content(content, table=table, parse_method=parse_method, url=url)

    def parse_list(self, table, parse_method='xpath', url=None):
        parse_method = f"get_by_{parse_method}"
        getattr(self, parse_method)('list', self.field_item['list'].get('value'))
        for item in self.element_list:
            print("item:", item)
            self.parse_content(table=table, parse_method=parse_method, each_element=item, url=url)
            if self.is_empty_data():
                self.db.insert_one(table, columns_dict=self.data)
        self.duplicate.check_same_url_save(url)

    def parse_content(self, content=None, table=None, parse_method='xpath', each_element=None, url=None):
        deny_key = ("list",)
        print(parse_method)
        if content:
            parse_method = f"get_by_{parse_method}"
            for key, value in self.field_item.items():
                print("item:", key, value)
                getattr(self, parse_method)(key, value.get('value'), self.content)
            self.db.update(table, update_dict=self.data, condition_dict={
                "next_url": url
            })
            self.duplicate.check_same_url_save(url)
            self.data = {}
        else:
            if not self.next_url_rule:
                self.data['next_url'] = None
            else:
                # getattr(self, parse_method)("next_url", self.next_url_rule, each_element)
                self.get_next_url(parse_method=parse_method, url=url, element=each_element)
            self.data['next_layout_id'] = self.next_layout_info[0] if self.next_layout_info else -1
            # print("data:", self.data)
            for key, value in self.field_item.items():
                if key not in deny_key:
                    getattr(self, parse_method)(key, value.get('value'), each_element)

    def get_next_url(self, parse_method=None, url=None, element=None):
        rule_list = self.next_url_rule.split(",")
        if len(rule_list) == 2:
            getattr(self, parse_method)("next_url", rule_list[1], element)
            print(url)
            self.join_url(rule_list, url, element)
            print("data:", self.data)

        elif len(rule_list) == 1:
            getattr(self, parse_method)("next_url", self.next_url_rule, element)

    def join_url(self, rule_list=[], url=None, element=None):
        first_url = rule_list[0]
        rule = {
            "url": url,
            "domain": self.app_info[3]
        }
        if self.data.get('next_url'):
            self.data['next_url'] = os.path.join(rule.get(first_url), self.data['next_url'])

    def is_empty_data(self):  # 判断插入的数据是否为空
        count = 0
        for value in self.data.values():
            if value is not None:
                count += 1
        return count > 2

    def parse_text_by_xpath(self, key, content_list):
        self.data[key] = "\n".join(content_list)

    def parse_html_by_xpath(self, key, value):
        encoding = self.app_info[6]
        element = self.content.xpath(value)[0]
        print("element:", element)
        content = etree.tostring(element, method='html', encoding=encoding)
        content = unescape(content.decode(encoding))
        self.data[key] = content.replace("'", "")

    # 通过xpath来解析
    def get_by_xpath(self, key, value, element=None):
        xpath_dict = self.field_item.get(key)  # 得到解析列表的规则
        field_type = "string"
        if xpath_dict:
            field_type = "list" if key == "list" else xpath_dict.get('type')
            content_list = element.xpath(value) if element else []

        if field_type == "text":
            self.parse_text_by_xpath(key, content_list)
        elif "html" in field_type:
            self.parse_html_by_xpath(key, value)
        elif field_type == "list":
            self.element_list = self.content.xpath(value)  # 解析出元素列表
        else:
            self.data[key] = element.xpath(value).pop(0) if len(element.xpath(value)) else None  # 得到解析的值
        if xpath_dict and xpath_dict.get('filter'):
            self.data[key] = self.filter.filter(self.data[key], xpath_dict.get('filter'))
        if field_type != "list":
            self.data[key] = self.filter.del_illegal_str(self.data[key])
        # print(self.data)
        # '''下载图片'''
        # if field_type == "img":
        #     self.download_picture(url=self.data[key])

    def get_by_json(self, key, value, element=None):
        xpath_dict = self.field_item.get(key)  # 得到解析列表的规则
        if xpath_dict and xpath_dict.get('type') == "list":
            self.element_list = jsonpath(self.content, value)
        else:
            data_list = jsonpath(element, value)
            self.data[key] = data_list[0] if len(data_list) else data_list

    def get_by_xml(self, key, value):
        pass

    def element_to_html(self, element):
        content = etree.tostring(element, method='html', encoding="utf-8")
        content = unescape(content.decode("utf-8"))
        return content

    def download_picture(self, url):
        return self.img_util.download(url)


if __name__ == '__main__':
    with open("../manager/test.html", "r") as fp:
        html = fp.read()
        html = etree.HTML(html)
        print(html.xpath('//*[@id="tlist"]/ul/li'))
